package cn.sh.tools.network;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.downloader.HttpClientDownloader;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.processor.PageProcessor;

import java.util.Set;
import java.util.stream.Collectors;

/**
 * @author code4crafter@gmail.com <br>
 * @since 0.5.1
 */
public class ImgPageSecondProcessor implements PageProcessor {

    private Site site = Site.me().setRetryTimes(3).setSleepTime(1000).setTimeOut(10000);

    @Override
    public Site getSite() {
        return site;
    }

    @Override
    public void process(Page page) {
        //Set<String> set = new HashSet<>(page.getHtml().links().regex("(https://www.1lou.me/thread-\\d+.htm)").all());
        //set.forEach(System.out::println);

        System.out.println("==================================================");

        // 1-
        Set<String> hrefSet = page.getHtml().getDocument().body().getElementsByClass("text-title")
                .stream()
                .filter(element -> element.html().contains(ImgPageConstant.SEARCH_KEY) ||
                        element.text().contains(ImgPageConstant.SEARCH_KEY))
                .map(element -> ImgPageConstant.URL_PREFIX + element.attr("href"))
                .collect(Collectors.toSet());

        if (hrefSet.size() == 0) {
            return;
        }

        // 2-
        HttpClientDownloader httpClientDownloader = new HttpClientDownloader();
        hrefSet.forEach(href -> {
            Set<String> imgHrefSet = httpClientDownloader.download(href).getDocument().body().getElementsByTag("img")
                    .stream()
                    .filter(element -> {
                        String titleValue = element.attr("title");
                        if (null != titleValue && !"".equals(titleValue)) {
                            String imgSuffix = titleValue.substring(titleValue.lastIndexOf(".") + 1).toLowerCase();
                            return ImgPageConstant.IMG_SUFFIX_SET.contains(imgSuffix);
                        }
                        return false;
                    })
                    .map(element -> ImgPageConstant.URL_PREFIX + element.attr("src"))
                    .collect(Collectors.toSet());

            // 3-
            imgHrefSet.forEach(imgHref -> Spider.create(new ImgPageEndProcessor())
                    .addUrl(imgHref)
                    .addPipeline(new ConsolePipeline())
                    .thread(5)
                    .run());
        });
    }

}
